{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pong - 强化学习\n",
    "\n",
    "如果在远程AWS EC2 GPU服务器上运行此笔记本，首先需要安装xvfb、mesa和egl库。假设服务器运行的是Ubuntu18.04，您可以通过以下方式实现:\n",
    "\n",
    "    sudo apt-get update -y  \n",
    "    sudo apt-get install -y mesa-utils libegl1-mesa xvfb freeglut3-dev\n",
    "\n",
    "然后使用命令启动屏幕会话管理程序(**译者注**:你也可以替换为tmux):\n",
    "\n",
    "    screen\n",
    "\n",
    "如果ssh会话断开，这将阻止笔记本退出。一旦进入屏幕会话，用下面代码启动笔记本环境:\n",
    "\n",
    "    jupyter notebook --no-browser --ip=<EXTERNAL_IP_ADDRESS>\n",
    "\n",
    "在运行Ubuntu18.04且有更新nvidia驱动程序的EC2服务器上，笔记本应该可以正常运行。然而，在较旧的操作系统或过时的驱动程序上，您可能会遇到很多与opengl的无头(\"headless\")渲染相关的问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p0 = os.path.abspath('.')\n",
    "p1 = os.path.abspath(os.path.join(p0, '..'))\n",
    "\n",
    "sys.path.insert(0, p1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jupylet.rl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PIL.Image\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_images(iml, flip=True, columns=4, figsize=(17, 8)):\n",
    "    \n",
    "    iml = list(iml)\n",
    "    \n",
    "    plt.figure(figsize=figsize)\n",
    "    for i, image in enumerate(iml):\n",
    "        \n",
    "        if flip and isinstance(image, np.ndarray):\n",
    "            image = image[::-1]\n",
    "        else:\n",
    "            image = image.transform(PIL.Image.FLIP_TOP_BOTTOM)\n",
    "        \n",
    "        plt.subplot(len(iml) // columns + 1, columns, i + 1)\n",
    "        plt.axis('off')\n",
    "        plt.imshow(image)\n",
    "        \n",
    "\n",
    "def show_image(image, flip=True):\n",
    "    \n",
    "    if isinstance(image, np.ndarray):\n",
    "        image = PIL.Image.fromarray(image)\n",
    "        \n",
    "    if flip:\n",
    "        image = image.transpose(PIL.Image.FLIP_TOP_BOTTOM)\n",
    "        \n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.getpid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pong = jupylet.rl.GameProcess('pong')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%time pong.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pong.call('save', 'pong-start.state')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit -n100 _ = pong.step(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_image(pong.observe()['screen0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "games = jupylet.rl.Games(['pong'] * 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "games.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_images(o['screen0'] for o in games.observe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "player0 = np.random.randint(0, 2, (len(games.games), 5))\n",
    "player0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "player1 = np.random.randint(0, 2, (len(games.games), 5))\n",
    "player1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit -n200 games.step(player0, player1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sl = games.step(player0, player1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "al = [s.pop('screen0') for s in sl]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_images(al)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_images(o['screen0'] for o in games.reset())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
